package main.pmc;

import cn.hutool.core.util.ZipUtil;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import cn.hutool.db.ds.DSFactory;
import lombok.SneakyThrows;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import utils.SmallTool;
import utils.StringUtil;

import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * @Author: gcr
 * @Date: 2022/3/12 13:20
 */
public class DownPmcidHtmlParseTestV2 {
    // 用于下载pmcid的页面保存


    @SneakyThrows
    public static void main(String[] args) {
//        DSFactory.setCurrentDSFactory(new DruidDSFactory());
//        DataSource ds = DSFactory.get("taiyiserver");
        Db useTaiyi = Db.use(DSFactory.get("local_articleadd"));
        String saveTableFrom = "pmcdownziphtml";
        String allDate = "select max(pmid) as pmid  from " + saveTableFrom;
        Entity query = useTaiyi.query(allDate).get(0);
        SmallTool.printMess("最大的id 是" + query.getStr("pmid"));
        Integer count = query.getInt("pmid");
        int breakSize = 100000;
        int pageSize = count / breakSize + 1;
        SmallTool.printMess("全部页码+++++++++" + pageSize);
        SmallTool.printMess("任务开始+++++++++++++++++++++++++++++++++++++++++++++++++++++");
        ThreadPoolExecutor executor = new ThreadPoolExecutor(
                100,
                200,
                10,
                TimeUnit.SECONDS,
                new LinkedBlockingQueue<>(), new ThreadPoolExecutor.CallerRunsPolicy());
        // 35000001
        for (int i = 1; i <= pageSize; i++) {
            // 这里写错了 需要只需要入库特定的数据 其他不要
            int start = breakSize * (i - 1) + 1;
            int end = i * breakSize;
            if (i <= 350) {
                continue;
            }
//            SmallTool.printMess("当前页码+++++" + i);
            SmallTool.printMess("开始id++" + start + "+++结束id++++" + end);
            String paramSql = "select pmid,pmcid,html from " + saveTableFrom + " where  pmid between  " + start + " and " + end;
//            String paramSql = "select pmid,pmcid,html,supp,down from " + saveTableFrom + " where  pmid  = 39901448 ";
            List<Entity> querys = useTaiyi.query(paramSql);
            querys.forEach(entity -> {
                executor.submit(() -> {
                    int pmid = entity.getInt("pmid");
//                    System.out.println("查询的pmid" + pmid);
                    byte[] htmls = entity.getBytes("html");
                    if (StringUtil.isNotNull(htmls)) {
                        String html = ZipUtil.unGzip(htmls, "UTF-8");
                        Document parse = Jsoup.parse(html);
                        Element element = parse.selectFirst("div:containsOwn(PMCID:)");
                        if (element != null) {
                            String text = element.text();
                            int pmidRealPmid = 0;
                            try {
                                pmidRealPmid = Integer.parseInt(text.split("PMID:")[1].trim());
                            } catch (Exception e) {
//                                e.printStackTrace();
                            }
                            if (pmidRealPmid != 0 && pmidRealPmid != pmid) {
                                SmallTool.printMess("pmid 不一致" + pmid + "---" + pmidRealPmid);
                                entity.set("real_pmid", pmidRealPmid);
                                try {
                                    useTaiyi.insertOrUpdate(entity, "pmid");
                                } catch (Exception e) {
                                    e.printStackTrace();
                                }
                            }
                        }
                    }
                });
            });

        }
        executor.shutdown();
        while (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
            int activeCount = executor.getActiveCount();
            System.out.println("当前线程数" + activeCount);
            long completedTaskCount = executor.getCompletedTaskCount();
            System.out.println("已完成任务数" + completedTaskCount);
        }
        System.out.println("任务结束");
    }
}
